package jarvis.utils;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.tika.Tika;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;

public class DataExtractionUtil {

	public static List<FileData> getFileDataFromFolder(String folder)
			throws IOException, SAXException, TikaException {
		List<FileData> list = new ArrayList<FileData>();
		File f = new File(folder);
		if (f.isDirectory() && !f.getName().contains("svn")) {
			File[] files = f.listFiles();
			for (int i = 0; i < files.length; i++) {
				list.addAll(getFileDataFromFolder(files[i].getCanonicalPath()));
			}
		} else if (!f.getName().contains("svn")) {
			list.add(getFileData(folder));
		}
		return list;
	}

	public static FileData getFileData(String filePath) throws IOException,
			SAXException, TikaException {
		HashMap metadataMap = new HashMap();
		InputStream input = new FileInputStream(filePath);
		// ContentHandler handler = new BodyContentHandler();
		ContentHandler handler = new BodyContentHandler(100000 * 100000);
		Metadata metadata = new Metadata();
		Parser parser = new AutoDetectParser();
		final FileData fileData = new FileData();
		String mimeType = new Tika().detect(filePath);
		fileData.setName(new File(filePath).getName());
		metadata.set(Metadata.CONTENT_TYPE, mimeType);
		try {
			parser.parse(input, handler, metadata, new ParseContext());
			fileData.setTextContent(handler.toString().replaceAll("\\s+", " "));
			if (mimeType.equals("application/xml")) {
				String plainText = getPlainTextFromFile(filePath);
				fileData.setRawtext(plainText.replaceAll("\\s+", " "));
			} else if (mimeType.equalsIgnoreCase("application/vnd.openxmlformats-officedocument.wordprocessingml.document")) {
				FileSystem fs = FileSystems.newFileSystem(Paths.get(filePath), null);
				fileData.setValidDocument(false);
				Files.walkFileTree(fs.getPath("word/media"), new FileVisitor<Path>() {

					public FileVisitResult postVisitDirectory(Path dir,
							IOException exc) throws IOException {
						return FileVisitResult.CONTINUE;
					}

					public FileVisitResult preVisitDirectory(Path dir,
							BasicFileAttributes attrs) throws IOException {
						return FileVisitResult.CONTINUE;
					}

					public FileVisitResult visitFile(Path file,
							BasicFileAttributes attrs) throws IOException {
						fileData.setValidDocument(true);
						return FileVisitResult.CONTINUE;
					}

					public FileVisitResult visitFileFailed(Path file,
							IOException exc) throws IOException {
						return FileVisitResult.CONTINUE;
					}
				});
			}
		} catch (TikaException e) {
			// check if file is XML
			// parse invaild XML file
			System.out.println(mimeType);
			if (mimeType.equals("application/xml")) {
				String plainText = getPlainTextFromFile(filePath);
				fileData.setRawtext(plainText.replaceAll("\\s+", " "));
				String processed = plainText.replaceAll("<[^>]+>", "");
				System.out.println(processed);
				fileData.setTextContent(processed);
				fileData.setValidDocument(false);
			}
		}

		input.close();

		for (int i = 0; i < metadata.names().length; i++) {
			String key = metadata.names()[i];
			metadataMap.put(key, metadata.get(key));
		}

		fileData.setMetadata(metadataMap);
		return fileData;
	}

	public static String getPlainTextFromFile(String filePath)
			throws IOException {
		InputStream input = new FileInputStream(filePath);
		StringBuilder builder = new StringBuilder();
		int ch;
		while ((ch = input.read()) != -1) {
			builder.append((char) ch);
		}

		input.close();
		return builder.toString();
	}

	public static void main(String args[]) {
		Client client = ESClientFactory.getClient();
		try {
			DeleteByQueryResponse response = client
					.prepareDeleteByQuery("jarvis")
					.setQuery(new TermQueryBuilder("_type", "filedata"))
					.execute().actionGet();
		} catch (Exception arve) {
			// arve.printStackTrace();
		}
		String filepath = "src\\test\\resources";
		DataExtractionUtil util = new DataExtractionUtil();
		try {
			List<FileData> data = DataExtractionUtil
					.getFileDataFromFolder(filepath);
			Map<String, Object> map = new HashMap<String, Object>();
			for (FileData fd : data) {
				map = (Map<String, Object>) fd.getMetadata();
				map.put("text", fd.getTextContent());
				map.put("rawText", fd.getRawtext());
				map.put("mime-type",
						((HashMap) fd.getMetadata()).get("Content-Type"));
				map.put("title",
						((HashMap) fd.getMetadata()).get("Title"));
				map.put("validDoc", new Boolean(fd.isValidDocument()));
				map.put("name", fd.getName());

				IndexResponse response = client
						.prepareIndex("jarvis", "filedata").setSource(map)
						.execute().actionGet();
			}
		} catch (Exception e) {
		}
		client.close();
	}

	// //public static void main(String args[]) {
	// String filepath = "src\\test\\resources\\Invalid_a.xml";
	// try {
	// DataExtractionUtil.getFileData(filepath);
	// } catch (IOException e) {
	// // TODO Auto-generated catch block
	// e.printStackTrace();
	// } catch (SAXException e) {
	// // TODO Auto-generated catch block
	// e.printStackTrace();
	// } catch (TikaException e) {
	// // TODO Auto-generated catch block
	// e.printStackTrace();
	// }
	// }
}
